from pyquery import PyQuery as pq
from requests.exceptions import RequestException
import requests
import re,json,time

def getPage(url):
    try: 
        headers = {
            'User-Agent':'User-Agent:Mozilla/5.0(WindowsNT6.1;rv:2.0.1)Gecko/20100101Firefox/4.0.1'
        }
        res = requests.get(url,headers=headers)
        if res.status_code==200:
            return res.content.decode("utf-8")
        else:
            return None
    except RequestException:
        return None

def parsePage(content):
    doc = pq(content)
    items = doc("tr.item")
    for item in items.items():
        yield{
            'title':item.find('div.pl2 a').text(),
            'info':item.find('p.pl').text(),
            'rating':item.find('span.rating_nums').text(),
            'comment':item.find('span.inq').text(),
        }

def writeFile(content):
    with open("./result.txt",'a',encoding='utf-8') as f:
        f.write(str(content)+"\n")

def main(offset):
    url = "https://book.douban.com/top250?start="+str(offset)
    html = getPage(url)
    if html:
        for item in parsePage(html):
            print(item)
            writeFile(item)

if __name__=="__main__":
    for i in range(10):
        main(offset=i*25)
        time.sleep(1)
